import requests
import json
import time
import pymongo

client = pymongo.MongoClient('localhost', 27017)
mydb = client['mydb']
lagou = mydb['lagou']

headers = {
'Host': 'www.lagou.com',
'Connection': 'keep-alive',
'Content-Length': '20',
'Origin': 'https://www.lagou.com',
'X-Anit-Forge-Code': '0',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'X-Requested-With': 'XMLHttpRequest',
'X-Anit-Forge-Token': 'None',
'Referer': 'https://www.lagou.com/jobs/list_?labelWords=&fromSearch=true&suginput=',
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'zh-CN,zh;q=0.9,und;q=0.8,en;q=0.7',
'Cookie': 'JSESSIONID=ABAAABAAAIAACBI8EB3BBB121C9AA99FC6B580150BC463D; Hm_lvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1547706307; _ga=GA1.2.1218967694.1547706307; user_trace_token=20190117142507-a246ef5b-1a20-11e9-b67c-5254005c3644; LGUID=20190117142507-a246f237-1a20-11e9-b67c-5254005c3644; index_location_city=%E5%85%A8%E5%9B%BD; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%221685a7c5650f4-0a80f84cafd84-10346655-1024000-1685a7c565167a%22%2C%22%24device_id%22%3A%221685a7c5650f4-0a80f84cafd84-10346655-1024000-1685a7c565167a%22%7D; _gid=GA1.2.214915758.1547797648; showExpriedIndex=1; showExpriedCompanyHome=1; showExpriedMyPublish=1; hasDeliver=9; LG_LOGIN_USER_ID=931dd48217d07b043b38ecf8c0e7ec9eee45fac65cbe9d26; _putrc=F13DDADB6F9A53A2; login=true; unick=%E4%BD%95%E8%8D%A3%E8%8D%A3; SEARCH_ID=c0e5ed66760c4226a44822897086f550; TG-TRACK-CODE=search_code; _gat=1; LGSID=20190118183324-7c038614-1b0c-11e9-b689-5254005c3644; PRE_UTM=; PRE_HOST=; PRE_SITE=; PRE_LAND=https%3A%2F%2Fwww.lagou.com%2F; gate_login_token=4d1f23fba34e79ec676ad534afd5286ba94c174f8215960e; LGRID=20190118183332-80c8db0f-1b0c-11e9-b689-5254005c3644; Hm_lpvt_4233e74dff0ae5bd0a3d81c6ccf756e6=1547807613'
}

def get_page(url,params):
    html = requests.post(url, data=params, headers=headers)
    json_data = json.loads(html.text)
    print(json_data)
    total_Count = json_data['content']['positionResult']['totalCount']
    page_number = int(total_Count/15) if int(total_Count/15)<30 else 30
    get_info(url,page_number)

def get_info(url,page):
    for pn in range(1,page+1):
        params = {
            'first': 'true',
            'pn': str(pn),
            'kd': 'Python'
        }
        try:
            html = requests.post(url,data=params,headers=headers)
            json_data = json.loads(html.text)
            results = json_data['content']['positionResult']['result']
            for result in results:
                infos = {
                    'businessZones':result['businessZones'],
                    'city':result['city'],
                    'companyFullName':result['companyFullName'],
                    'companyLabelList':result['companyLabelList'],
                    'companySize':result['companySize'],
                    'district':result['district'],
                    'education':result['education'],
                    'explain':result['explain'],
                    'financeStage':result['financeStage'],
                    'firstType':result['firstType'],
                    'formatCreateTime':result['formatCreateTime'],
                    'gradeDescription':result['gradeDescription'],
                    'imState':result['imState'],
                    'industryField':result['industryField'],
                    'jobNature':result['jobNature'],
                    'positionAdvantage':result['positionAdvantage'],
                    'salary':result['salary'],
                    'secondType':result['secondType'],
                    'workYear':result['workYear']
                }
                lagou.insert_one(infos)
                time.sleep(2)
        except requests.exceptions.ConnectionError:
            pass

if __name__ == '__main__':
    url = 'https://www.lagou.com/jobs/positionAjax.json'
    params = {
        'first': 'true',
        'pn': '1',
        'kd': 'Python'
    }
    get_page(url,params)